{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "vscode": {
     "languageId": "plaintext"
    }
   },
   "outputs": [],
   "source": [
    "# 导入数据集\n",
    "LENTH = 200\n",
    "\n",
    "# 读取数据集\n",
    "import pandas as pd\n",
    "\n",
    "file_path = \"hallucinated_answers.csv\"\n",
    "\n",
    "# 使用 pandas 读取 CSV 文件\n",
    "df = pd.read_csv(file_path)\n",
    "\n",
    "# 选择第一列（索引 0）、第四列（索引 3）和第六列（索引 5）\n",
    "df_selected = df.iloc[:LENTH, [0, 3, 5]]\n",
    "# 显示读取的数据\n",
    "print(df_selected.head())  # 打印前几行以确认数据是否成功读取"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "vscode": {
     "languageId": "plaintext"
    }
   },
   "outputs": [],
   "source": [
    "from google import genai\n",
    "\n",
    "import pandas as pd\n",
    "from openai import OpenAI,RateLimitError\n",
    "import random\n",
    "\n",
    "# 假设 df 已经加载，包含三列: 'question', 'hallucination_answer', 'correct_answer'\n",
    "\n",
    "# 初始化 DeepSeek API 客户端\n",
    "#client = OpenAI(api_key=\"sk-345d4d96bd464da2bfebb09bc43b4aa4\", base_url=\"https://api.deepseek.com\")\n",
    "\n",
    "# 随机数list\n",
    "# 生成一个随机数链表，长度为 1000，数值为 0 或 1\n",
    "random_linked_list = [random.choice([0, 1]) for _ in range(LENTH)]\n",
    "\n",
    "# 输出链表的前10个元素作为示例\n",
    "print(random_linked_list[:10])\n",
    "\n",
    "\n",
    "# 判断幻觉的函数\n",
    "def judge_hallucination(client,question, hallucination_answer): #这个是gemini专用，其他的写个openapi\n",
    "    # client = genai.Client(api_key=\"AIzaSyB2Atc4agblDdsqa3_xKh3_K81tr2OW0t8\")\n",
    "    # response = client.models.generate_content(\n",
    "    # model=\"gemini-2.0-flash\",\n",
    "    # contents=f\"你是 gemini，我下面会给你一个问题以及一个别的大语言模型给出的答案，请你判断这个回答是否出现幻觉现象，你只需要回答是或不是。问题为：{question}；答案为：{hallucination_answer}\",\n",
    "    # )\n",
    "    # client = OpenAI(api_key=\"sk-345d4d96bd464da2bfebb09bc43b4aa4\", base_url=\"https://api.deepseek.com\")\n",
    "\n",
    "    response = client.chat.completions.create(\n",
    "    model=\"deepseek-chat\",\n",
    "    #model=\"qwen-plus\",\n",
    "    messages=[\n",
    "        {\"role\": \"system\", \"content\": \"我下面会给你一个问题以及一个别的大语言模型给出的答案，请你判断这个回答是否出现幻觉现象，只需要回答是或不是,不要有多余的字符，只需要是，或者不是\"},\n",
    "        {\"role\": \"user\", \"content\": f\"问题为：{question};答案为：{hallucination_answer}\"}\n",
    "    ],\n",
    "    stream=False)\n",
    "\n",
    "\n",
    "    return response.choices[0].message.content.strip()\n",
    "\n",
    "# 计算裁判模型准确率\n",
    "# def evaluate_hallucination_accuracy(df):\n",
    "#     correct_count = 0\n",
    "#     total_count = len(df)\n",
    "\n",
    "#     for _, row in df.iterrows():\n",
    "#         question = row['Question']\n",
    "\n",
    "#         if random_linked_list[_]:\n",
    "#             # 给予错误答案\n",
    "#             hallucination_answer = row['halu']\n",
    "#             # 使用 DeepSeek 判断答案是否幻觉\n",
    "#             result = judge_hallucination(question, hallucination_answer)\n",
    "#             # 如果裁判模型的答案是 \"是\"，并且幻觉答案是错误的，那么算作正确的判断\n",
    "#             if result == \"是\":\n",
    "#                 correct_count += 1\n",
    "#         else:\n",
    "#             correct_answer = row['non-halu']\n",
    "#             # 使用 DeepSeek 判断答案是否幻觉\n",
    "#             result = judge_hallucination(question, correct_answer)\n",
    "#             # 如果裁判模型的答案是 \"是\"，并且幻觉答案是错误的，那么算作正确的判断\n",
    "#             if result == \"不是\":\n",
    "#                 correct_count += 1\n",
    "#         print(f'进行第{ _ +1}轮')\n",
    "#     accuracy = correct_count / total_count * 100\n",
    "\n",
    "#     return accuracy\n",
    "\n",
    "\n",
    "import random\n",
    "import time\n",
    "from google.genai.errors import ClientError\n",
    "\n",
    "# api1 = \"AIzaSyB2Atc4agblDdsqa3_xKh3_K81tr2OW0t8\"\n",
    "# api2 = \"AIzaSyAaxF2OcGHPZJWx5wlzhIewdzxzCM3n5OY\"\n",
    "\n",
    "\n",
    "def evaluate_hallucination_accuracy(api,df, random_linked_list,correct_count,count,LENTH):\n",
    "    total_count = LENTH\n",
    "\n",
    "    #client = genai.Client(api_key=api)\n",
    "    client = OpenAI(api_key=\"sk-345d4d96bd464da2bfebb09bc43b4aa4\", base_url=\"https://api.deepseek.com\")\n",
    "#     client = OpenAI(\n",
    "#     api_key=\"sk-fb3964c128a74650be4c226d9b7dd9bc\",\n",
    "#     base_url=\"https://dashscope.aliyuncs.com/compatible-mode/v1\",\n",
    "# )\n",
    "\n",
    "    # 确保 random_linked_list 长度与 df 相同\n",
    "    # assert len(random_linked_list) == total_count, \"随机链表长度与数据集行数不一致！\"\n",
    "\n",
    "    old_count = count\n",
    "    # 使用 enumerate 来获取索引和数据行\n",
    "    for idx in range(old_count,LENTH+1):\n",
    "\n",
    "        question = df['question'][idx]\n",
    "        print(question,\"\\n\")\n",
    "        try:\n",
    "            # 根据 random_linked_list 决定使用幻觉答案还是正确答案\n",
    "            if random_linked_list[idx]:\n",
    "                # 给予错误答案（幻觉答案）\n",
    "                hallucination_answer = df['hallucinated_answer'][idx]\n",
    "                # 使用 DeepSeek 判断答案是否幻觉\n",
    "                result = judge_hallucination(client,question, hallucination_answer)\n",
    "                result = result.strip()\n",
    "                # 如果裁判模型的答案是 \"是\"，并且幻觉答案是错误的，那么算作正确的判断\n",
    "                if result == \"是\":\n",
    "                    correct_count += 1\n",
    "                # 打印当前进度\n",
    "                print(f'正确答案为：是，回答为：{result}，进行第 {idx + 1} 轮，正确个数为{correct_count}')\n",
    "\n",
    "            else:\n",
    "                # 给予正确答案\n",
    "                correct_answer = df['right_answer'][idx]\n",
    "                # 使用 DeepSeek 判断答案是否幻觉\n",
    "                result = judge_hallucination(client,question, correct_answer)\n",
    "                result = result.strip()\n",
    "                # 如果裁判模型的答案是 \"不是\"，并且幻觉答案是正确的，那么算作正确的判断\n",
    "                if result == \"不是\":\n",
    "                    correct_count += 1\n",
    "                # 打印当前进度\n",
    "                print(f'正确答案为：不是，回答为：{result}，进行第 {count+1} 轮，正确个数为{correct_count}')\n",
    "            # 延迟 1 秒，以控制请求频率\n",
    "            #time.sleep(0.5)\n",
    "            count+=1\n",
    "\n",
    "        # except ClientError as e:\n",
    "        #   if e.code == 429:\n",
    "        #       print(\"API 请求配额已用尽，正在等待 30 秒后重试...\")\n",
    "        #       time.sleep(5)  # 等待 60 秒后重试\n",
    "        #       if api == api1:\n",
    "        #         return evaluate_hallucination_accuracy(api2,df_selected, random_linked_list,correct_count,count,LENTH)  # 递归重试\n",
    "        #       else:\n",
    "        #         return evaluate_hallucination_accuracy(api1,df_selected, random_linked_list,correct_count,count,LENTH)  # 递归重试\n",
    "        #       # 这里想继续进程，那么我们要记录进程，然后嵌套函数执行\n",
    "        #       # return correct_count / idx *100\n",
    "        except RateLimitError as e:\n",
    "          print(\"API 请求配额已用尽，正在等待 30 秒后重试...\")\n",
    "          time.sleep(5)  # 等待 60 秒后重试\n",
    "          # if api == api1:\n",
    "          #   return evaluate_hallucination_accuracy(api2,df_selected, random_linked_list,correct_count,count,LENTH)  # 递归重试\n",
    "          # else:\n",
    "          return evaluate_hallucination_accuracy(\"sk-345d4d96bd464da2bfebb09bc43b4aa4\",df_selected, random_linked_list,correct_count,count,LENTH)  # 递归重试\n",
    "          # 这里想继续进程，那么我们要记录进程，然后嵌套函数执行\n",
    "          # return correct_count / idx *100\n",
    "\n",
    "    # 计算准确率\n",
    "    accuracy = correct_count / total_count * 100\n",
    "\n",
    "    return accuracy\n",
    "\n",
    "# 假设 df 已经加载，调用评估函数\n",
    "accuracy = evaluate_hallucination_accuracy(\"sk-345d4d96bd464da2bfebb09bc43b4aa4\",df_selected,random_linked_list,0,0,LENTH)\n",
    "print(f\"裁判模型的准确率是: {accuracy}%\")\n"
   ]
  }
 ],
 "metadata": {
  "language_info": {
   "name": "python"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}
